diff --git a/etc/install-libmongocrypt.sh b/etc/install-libmongocrypt.sh index a94d648eae..9905cd62f1 100755 --- a/etc/install-libmongocrypt.sh +++ b/etc/install-libmongocrypt.sh @@ -3,7 +3,7 @@ # This script installs libmongocrypt into an "install" directory. set -eux -LIBMONGOCRYPT_TAG="1.12.0" +LIBMONGOCRYPT_TAG="1.15.1" # Install libmongocrypt based on OS. if [ "Windows_NT" = "${OS:-}" ]; then diff --git a/internal/integration/mtest/mongotest.go b/internal/integration/mtest/mongotest.go index 3924b58604..dbd69da3b9 100644 --- a/internal/integration/mtest/mongotest.go +++ b/internal/integration/mtest/mongotest.go @@ -797,12 +797,6 @@ func verifyRunOnBlockConstraint(rob RunOnBlock) error { return err } - // TODO(GODRIVER-3486): Once auto encryption is supported by the unified test - // format,this check should be removed. - if rob.CSFLEEnabled() && rob.CSFLE.Options != nil { - return fmt.Errorf("Auto encryption required (GODRIVER-3486)") - } - if rob.CSFLEEnabled() && !IsCSFLEEnabled() { return fmt.Errorf("runOnBlock requires CSFLE to be enabled. Build with the cse tag to enable") } else if !rob.CSFLEEnabled() && IsCSFLEEnabled() { diff --git a/internal/integration/unified/client_entity.go b/internal/integration/unified/client_entity.go index bc981793df..f2171395a9 100644 --- a/internal/integration/unified/client_entity.go +++ b/internal/integration/unified/client_entity.go @@ -8,6 +8,7 @@ package unified import ( "context" + "crypto/tls" "fmt" "strings" "sync" @@ -217,6 +218,13 @@ func newClientEntity(ctx context.Context, em *EntityMap, entityOptions *entityOp } else { integtest.AddTestServerAPIVersion(clientOpts) } + if entityOptions.AutoEncryptOpts != nil { + aeo, err := createAutoEncryptionOptions(entityOptions.AutoEncryptOpts) + if err != nil { + return nil, fmt.Errorf("error parsing auto encryption options: %w", err) + } + clientOpts.SetAutoEncryptionOptions(aeo) + } for _, cmd := range entityOptions.IgnoredCommands { entity.ignoredCommands[cmd] = struct{}{} } @@ -251,6 +259,82 @@ func getURIForClient(opts *entityOptions) string { } } +func createAutoEncryptionOptions(opts bson.Raw) (*options.AutoEncryptionOptions, error) { + aeo := options.AutoEncryption() + var kvnsFound bool + elems, err := opts.Elements() + if err != nil { + return nil, err + } + + for _, elem := range elems { + name := elem.Key() + opt := elem.Value() + + switch name { + case "kmsProviders": + providers := make(map[string]map[string]any) + elems, err := opt.Document().Elements() + if err != nil { + return nil, err + } + for _, elem := range elems { + key := elem.Key() + opt := elem.Value().Document() + provider, err := getKmsProvider(key, opt) + if err != nil { + return nil, err + } + if provider == nil { + continue + } + providers[key] = provider + if key == "kmip" && tlsClientCertificateKeyFile != "" && tlsCAFile != "" { + cfg, err := options.BuildTLSConfig(map[string]any{ + "tlsCertificateKeyFile": tlsClientCertificateKeyFile, + "tlsCAFile": tlsCAFile, + }) + if err != nil { + return nil, fmt.Errorf("error constructing tls config: %w", err) + } + aeo.SetTLSConfig(map[string]*tls.Config{ + "kmip": cfg, + }) + } + } + aeo.SetKmsProviders(providers) + case "schemaMap": + var schemaMap map[string]any + err := bson.Unmarshal(opt.Document(), &schemaMap) + if err != nil { + return nil, fmt.Errorf("error creating schema map: %v", err) + } + aeo.SetSchemaMap(schemaMap) + case "keyVaultNamespace": + kvnsFound = true + aeo.SetKeyVaultNamespace(opt.StringValue()) + case "bypassAutoEncryption": + aeo.SetBypassAutoEncryption(opt.Boolean()) + case "encryptedFieldsMap": + var encryptedFieldsMap map[string]any + err := bson.Unmarshal(opt.Document(), &encryptedFieldsMap) + if err != nil { + return nil, fmt.Errorf("error creating encryptedFieldsMap: %v", err) + } + aeo.SetEncryptedFieldsMap(encryptedFieldsMap) + case "bypassQueryAnalysis": + aeo.SetBypassQueryAnalysis(opt.Boolean()) + default: + return nil, fmt.Errorf("unrecognized option: %v", name) + } + } + if !kvnsFound { + aeo.SetKeyVaultNamespace("keyvault.datakeys") + } + + return aeo, nil +} + // disconnect disconnects the client associated with this entity. It is an // idempotent operation, unlike the mongo client's disconnect method. This // property will help avoid unnecessary errors when calling disconnect on a diff --git a/internal/integration/unified/collection_data.go b/internal/integration/unified/collection_data.go index 02ceb3f147..3fc6213875 100644 --- a/internal/integration/unified/collection_data.go +++ b/internal/integration/unified/collection_data.go @@ -27,8 +27,10 @@ type collectionData struct { } type createOptions struct { - Capped *bool `bson:"capped"` - SizeInBytes *int64 `bson:"size"` + Capped *bool `bson:"capped"` + SizeInBytes *int64 `bson:"size"` + EncryptedFields bson.Raw `bson:"encryptedFields"` + Validator bson.Raw `bson:"validator"` } // createCollection configures the collection represented by the receiver using the internal client. This function @@ -49,14 +51,18 @@ func (c *collectionData) createCollection(ctx context.Context) error { if c.Options.SizeInBytes != nil { createOpts = createOpts.SetSizeInBytes(*c.Options.SizeInBytes) } + if c.Options.EncryptedFields != nil { + createOpts = createOpts.SetEncryptedFields(c.Options.EncryptedFields) + } + if c.Options.Validator != nil { + createOpts = createOpts.SetValidator(c.Options.Validator) + } if err := db.CreateCollection(ctx, c.CollectionName, createOpts); err != nil { return fmt.Errorf("error creating collection: %w", err) } - } - - // If neither documents nor options are provided, still create the collection with write concern "majority". - if len(c.Documents) == 0 && c.Options == nil { + } else { + // If options are provided, still create the collection with write concern "majority". // The write concern has to be manually specified in the command document because RunCommand does not honor // the database's write concern. create := bson.D{ @@ -68,13 +74,15 @@ func (c *collectionData) createCollection(ctx context.Context) error { if err := db.RunCommand(ctx, create).Err(); err != nil { return fmt.Errorf("error creating collection: %w", err) } - return nil } - docs := bsonutil.RawToInterfaces(c.Documents...) - if _, err := coll.InsertMany(ctx, docs); err != nil { - return fmt.Errorf("error inserting data: %w", err) + if len(c.Documents) != 0 { + docs := bsonutil.RawToInterfaces(c.Documents...) + if _, err := coll.InsertMany(ctx, docs); err != nil { + return fmt.Errorf("error inserting data: %w", err) + } } + return nil } diff --git a/internal/integration/unified/database_operation_execution.go b/internal/integration/unified/database_operation_execution.go index 76b215f833..940bf37335 100644 --- a/internal/integration/unified/database_operation_execution.go +++ b/internal/integration/unified/database_operation_execution.go @@ -125,6 +125,10 @@ func executeCreateCollection(ctx context.Context, operation *operation) (*operat cco.SetTimeSeriesOptions(tso) case "clusteredIndex": cco.SetClusteredIndex(val.Document()) + case "validator": + cco.SetValidator(val.Document()) + case "encryptedFields": + cco.SetEncryptedFields(val.Document()) default: return nil, fmt.Errorf("unrecognized createCollection option %q", key) } @@ -156,6 +160,8 @@ func executeDropCollection(ctx context.Context, operation *operation) (*operatio return nil, err } + dco := options.DropCollection() + var collName string elems, _ := operation.Arguments.Elements() for _, elem := range elems { @@ -163,6 +169,8 @@ func executeDropCollection(ctx context.Context, operation *operation) (*operatio val := elem.Value() switch key { + case "encryptedFields": + dco.SetEncryptedFields(val.Document()) case "collection": collName = val.StringValue() default: @@ -173,7 +181,7 @@ func executeDropCollection(ctx context.Context, operation *operation) (*operatio return nil, newMissingArgumentError("collection") } - err = db.Collection(collName).Drop(ctx) + err = db.Collection(collName).Drop(ctx, dco) return newErrorResult(err), nil } diff --git a/internal/integration/unified/entity.go b/internal/integration/unified/entity.go index b1b827a124..eec125388c 100644 --- a/internal/integration/unified/entity.go +++ b/internal/integration/unified/entity.go @@ -33,6 +33,8 @@ var ( tlsClientCertificateKeyFile = os.Getenv("CSFLE_TLS_CLIENT_CERT_FILE") ) +var placeholderDoc = bsoncore.NewDocumentBuilder().AppendInt32("$$placeholder", 1).Build() + type storeEventsAsEntitiesConfig struct { EventListID string `bson:"id"` Events []string `bson:"events"` @@ -52,6 +54,7 @@ type entityOptions struct { ID string `bson:"id"` // Options for client entities. + AutoEncryptOpts bson.Raw `bson:"autoEncryptOpts"` URIOptions bson.M `bson:"uriOptions"` UseMultipleMongoses *bool `bson:"useMultipleMongoses"` ObserveEvents []string `bson:"observeEvents"` @@ -503,6 +506,12 @@ func (em *EntityMap) close(ctx context.Context) []error { } } + for id, db := range em.dbEntites { + if err := db.Drop(ctx); err != nil { + errs = append(errs, fmt.Errorf("error dropping database with ID %q: %w", id, err)) + } + } + for id, client := range em.clientEntities { if ok := em.keyVaultClientIDs[id]; ok { // Client will be closed in clientEncryption.Close() @@ -563,13 +572,13 @@ func (em *EntityMap) addDatabaseEntity(entityOptions *entityOptions) error { // getKmsCredential processes a value of an input KMS provider credential. // An empty document returns from the environment. // A string is returned as-is. -func getKmsCredential(kmsDocument bson.Raw, credentialName string, envVar string, defaultValue string) (string, error) { +func getKmsCredential(kmsDocument bson.Raw, credentialName string, envVar string, defaultValue string) (any, error) { credentialVal, err := kmsDocument.LookupErr(credentialName) if errors.Is(err, bsoncore.ErrElementNotFound) { - return "", nil + return nil, nil } if err != nil { - return "", err + return nil, err } if str, ok := credentialVal.StringValueOK(); ok { @@ -579,135 +588,135 @@ func getKmsCredential(kmsDocument bson.Raw, credentialName string, envVar string var ok bool var doc bson.Raw if doc, ok = credentialVal.DocumentOK(); !ok { - return "", fmt.Errorf("expected String or Document for %v, got: %v", credentialName, credentialVal) + return nil, fmt.Errorf("expected String or Document for %v, got: %v", credentialName, credentialVal) } - placeholderDoc := bsoncore.NewDocumentBuilder().AppendInt32("$$placeholder", 1).Build() - // Check if document is a placeholder. if !bytes.Equal(doc, placeholderDoc) { - return "", fmt.Errorf("unexpected non-empty document for %v: %v", credentialName, doc) + return nil, fmt.Errorf("unexpected non-empty document for %v: %v", credentialName, doc) } + if envVar == "" { return defaultValue, nil } - if os.Getenv(envVar) == "" { - if defaultValue != "" { - return defaultValue, nil - } - return "", fmt.Errorf("unable to get environment value for %v. Please set the CSFLE environment variable: %v", credentialName, envVar) + if value := os.Getenv(envVar); value != "" { + return value, nil } - return os.Getenv(envVar), nil - + if defaultValue != "" { + return defaultValue, nil + } + return nil, fmt.Errorf("unable to get environment value for %v. Please set the CSFLE environment variable: %v", credentialName, envVar) } -func (em *EntityMap) addClientEncryptionEntity(entityOptions *entityOptions) error { - // Construct KMS providers. - kmsProviders := make(map[string]map[string]any) - ceo := entityOptions.ClientEncryptionOpts - tlsconf := make(map[string]*tls.Config) - if aws, ok := ceo.KmsProviders["aws"]; ok { - kmsProviders["aws"] = make(map[string]any) +func getKmsProvider(key string, opt bson.Raw) (map[string]any, error) { + provider := make(map[string]any) + switch key { + case "aws": + accessKeyID := "FLE_AWS_KEY" + secretAccessKey := "FLE_AWS_SECRET" - awsSessionToken, err := getKmsCredential(aws, "sessionToken", "CSFLE_AWS_TEMP_SESSION_TOKEN", "") + // replace with temporary access, if sessionToken placeholder exists + v, err := getKmsCredential(opt, "sessionToken", "", "$$placeholder") if err != nil { - return err + return nil, err + } + if v == "$$placeholder" { + provider["sessionToken"] = os.Getenv("CSFLE_AWS_TEMP_SESSION_TOKEN") + accessKeyID = "CSFLE_AWS_TEMP_ACCESS_KEY_ID" + secretAccessKey = "CSFLE_AWS_TEMP_SECRET_ACCESS_KEY" + } else if v != nil { + provider["sessionToken"] = v } - if awsSessionToken != "" { - // Get temporary AWS credentials. - kmsProviders["aws"]["sessionToken"] = awsSessionToken - awsAccessKeyID, err := getKmsCredential(aws, "accessKeyId", "CSFLE_AWS_TEMP_ACCESS_KEY_ID", "") - if err != nil { - return err - } - if awsAccessKeyID != "" { - kmsProviders["aws"]["accessKeyId"] = awsAccessKeyID - } - awsSecretAccessKey, err := getKmsCredential(aws, "secretAccessKey", "CSFLE_AWS_TEMP_SECRET_ACCESS_KEY", "") + for _, e := range []struct { + key string + envVar string + }{ + {"accessKeyId", accessKeyID}, + {"secretAccessKey", secretAccessKey}, + } { + v, err = getKmsCredential(opt, e.key, e.envVar, "") if err != nil { - return err + return nil, err } - if awsSecretAccessKey != "" { - kmsProviders["aws"]["secretAccessKey"] = awsSecretAccessKey + if v != nil { + provider[e.key] = v } - } else { - awsAccessKeyID, err := getKmsCredential(aws, "accessKeyId", "FLE_AWS_KEY", "") + } + case "azure": + for _, e := range []struct { + key string + envVar string + }{ + {"tenantId", "FLE_AZURE_TENANTID"}, + {"clientId", "FLE_AZURE_CLIENTID"}, + {"clientSecret", "FLE_AZURE_CLIENTSECRET"}, + } { + v, err := getKmsCredential(opt, e.key, e.envVar, "") if err != nil { - return err + return nil, err } - if awsAccessKeyID != "" { - kmsProviders["aws"]["accessKeyId"] = awsAccessKeyID + if v != nil { + provider[e.key] = v } - - awsSecretAccessKey, err := getKmsCredential(aws, "secretAccessKey", "FLE_AWS_SECRET", "") + } + case "gcp": + for _, e := range []struct { + key string + envVar string + }{ + {"email", "FLE_GCP_EMAIL"}, + {"privateKey", "FLE_GCP_PRIVATEKEY"}, + } { + v, err := getKmsCredential(opt, e.key, e.envVar, "") if err != nil { - return err + return nil, err } - if awsSecretAccessKey != "" { - kmsProviders["aws"]["secretAccessKey"] = awsSecretAccessKey + if v != nil { + provider[e.key] = v } } - - } - - if azure, ok := ceo.KmsProviders["azure"]; ok { - kmsProviders["azure"] = make(map[string]any) - - azureTenantID, err := getKmsCredential(azure, "tenantId", "FLE_AZURE_TENANTID", "") - if err != nil { - return err - } - if azureTenantID != "" { - kmsProviders["azure"]["tenantId"] = azureTenantID - } - - azureClientID, err := getKmsCredential(azure, "clientId", "FLE_AZURE_CLIENTID", "") + case "kmip": + v, err := getKmsCredential(opt, "endpoint", "", "localhost:5698") if err != nil { - return err + return nil, err } - if azureClientID != "" { - kmsProviders["azure"]["clientId"] = azureClientID + if v != nil { + provider["endpoint"] = v } - - azureClientSecret, err := getKmsCredential(azure, "clientSecret", "FLE_AZURE_CLIENTSECRET", "") + case "local", "local:name2": + defaultLocalKeyBase64 := "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" + v, err := getKmsCredential(opt, "key", "", defaultLocalKeyBase64) if err != nil { - return err + return nil, err } - if azureClientSecret != "" { - kmsProviders["azure"]["clientSecret"] = azureClientSecret + if v != nil { + provider["key"] = v } + default: + return nil, fmt.Errorf("unrecognized KMS provider: %s", key) } - - if gcp, ok := ceo.KmsProviders["gcp"]; ok { - kmsProviders["gcp"] = make(map[string]any) - - gcpEmail, err := getKmsCredential(gcp, "email", "FLE_GCP_EMAIL", "") - if err != nil { - return err - } - if gcpEmail != "" { - kmsProviders["gcp"]["email"] = gcpEmail - } - - gcpPrivateKey, err := getKmsCredential(gcp, "privateKey", "FLE_GCP_PRIVATEKEY", "") - if err != nil { - return err - } - if gcpPrivateKey != "" { - kmsProviders["gcp"]["privateKey"] = gcpPrivateKey - } + if len(provider) == 0 { + return nil, nil } + return provider, nil +} - if kmip, ok := ceo.KmsProviders["kmip"]; ok { - kmsProviders["kmip"] = make(map[string]any) - - kmipEndpoint, err := getKmsCredential(kmip, "endpoint", "", "localhost:5698") +func (em *EntityMap) addClientEncryptionEntity(entityOptions *entityOptions) error { + // Construct KMS providers. + kmsProviders := make(map[string]map[string]any) + ceo := entityOptions.ClientEncryptionOpts + tlsconf := make(map[string]*tls.Config) + for key, opt := range ceo.KmsProviders { + provider, err := getKmsProvider(key, opt) if err != nil { return err } - - if tlsClientCertificateKeyFile != "" && tlsCAFile != "" { + if provider == nil { + continue + } + kmsProviders[key] = provider + if key == "kmip" && tlsClientCertificateKeyFile != "" && tlsCAFile != "" { cfg, err := options.BuildTLSConfig(map[string]any{ "tlsCertificateKeyFile": tlsClientCertificateKeyFile, "tlsCAFile": tlsCAFile, @@ -717,23 +726,6 @@ func (em *EntityMap) addClientEncryptionEntity(entityOptions *entityOptions) err } tlsconf["kmip"] = cfg } - - if kmipEndpoint != "" { - kmsProviders["kmip"]["endpoint"] = kmipEndpoint - } - } - - if local, ok := ceo.KmsProviders["local"]; ok { - kmsProviders["local"] = make(map[string]any) - - defaultLocalKeyBase64 := "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" - localKey, err := getKmsCredential(local, "key", "", defaultLocalKeyBase64) - if err != nil { - return err - } - if localKey != "" { - kmsProviders["local"]["key"] = localKey - } } em.keyVaultClientIDs[ceo.KeyVaultClient] = true diff --git a/internal/integration/unified/error.go b/internal/integration/unified/error.go index ca4e985433..c1f5ba04c3 100644 --- a/internal/integration/unified/error.go +++ b/internal/integration/unified/error.go @@ -42,7 +42,7 @@ type clientBulkWriteException struct { // will perform any other assertions required by the expectedError object. An error is returned if any checks fail. func verifyOperationError(ctx context.Context, expected *expectedError, result *operationResult) error { if expected == nil { - if result.Err != nil { + if result != nil && result.Err != nil { return fmt.Errorf("expected no error, but got %w", result.Err) } return nil diff --git a/internal/integration/unified/matches.go b/internal/integration/unified/matches.go index 6d34df1d95..8129f2188f 100644 --- a/internal/integration/unified/matches.go +++ b/internal/integration/unified/matches.go @@ -150,6 +150,19 @@ func verifyValuesMatchInner(ctx context.Context, expected, actual bson.RawValue) return nil } + if expected.Type == bson.TypeDecimal128 { + if actual.Type != bson.TypeDecimal128 { + return newMatchingError(keyPath, "expected value to be a decimal type but got a %s", actual.Type) + } + expectedDecimal := expected.Decimal128() + actualDecimal := actual.Decimal128() + eh, el := expectedDecimal.GetBytes() + ah, al := actualDecimal.GetBytes() + if eh != ah || el != al { + return newMatchingError(keyPath, "expected decimal value %v, got %v", expectedDecimal, actualDecimal) + } + return nil + } // Numeric values must be considered equal even if their types are different (e.g. if expected is an int32 and // actual is an int64). if expected.IsNumber() { diff --git a/internal/integration/unified/operation.go b/internal/integration/unified/operation.go index 1b591d66af..a6757abd1e 100644 --- a/internal/integration/unified/operation.go +++ b/internal/integration/unified/operation.go @@ -275,8 +275,19 @@ func (op *operation) run(ctx context.Context, loopDone <-chan struct{}) (*operat case "decrypt": return executeDecrypt(ctx, op) + case "assertIndexNotExists": + db := lookupString(op.Arguments, "databaseName") + coll := lookupString(op.Arguments, "collectionName") + index := lookupString(op.Arguments, "indexName") + return newErrorResult(nil), verifyIndexExists(ctx, db, coll, index, false) + case "assertIndexExists": + db := lookupString(op.Arguments, "databaseName") + coll := lookupString(op.Arguments, "collectionName") + index := lookupString(op.Arguments, "indexName") + return newErrorResult(nil), verifyIndexExists(ctx, db, coll, index, true) + // Unsupported operations - case "count", "listIndexNames": + case "count", "listIndexNames", "mapReduce": return nil, newSkipTestError(fmt.Sprintf("the %q operation is not supported", op.Name)) default: return nil, fmt.Errorf("unrecognized entity operation %q", op.Name) diff --git a/internal/spectest/skip.go b/internal/spectest/skip.go index 189b38b447..74a864d3fe 100644 --- a/internal/spectest/skip.go +++ b/internal/spectest/skip.go @@ -392,17 +392,10 @@ var skipTests = map[string][]string{ "TestClientSideEncryptionSpec/timeoutMS.json/timeoutMS_applied_to_listCollections_to_get_collection_schema", }, - // TODO(GODRIVER-3486): Support auto encryption in unified tests. - "Support auto encryption in unified tests (GODRIVER-3486)": { - "TestUnifiedSpec/unified-test-format/tests/valid-pass/poc-queryable-encryption.json/insert,_replace,_and_find_with_queryable_encryption", - }, - - // TODO(DRIVERS-3106): Support auto encryption in unified tests. - "Support auto encryption in unified tests (DRIVERS-3106)": { - "TestUnifiedSpec/client-side-encryption/tests/unified/localSchema.json/A_local_schema_should_override", - "TestUnifiedSpec/client-side-encryption/tests/unified/localSchema.json/A_local_schema_with_no_encryption_is_an_error", - "TestUnifiedSpec/client-side-encryption/tests/unified/fle2v2-BypassQueryAnalysis.json/BypassQueryAnalysis_decrypts", - "TestUnifiedSpec/client-side-encryption/tests/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json/encryptedFieldsMap_is_preferred_over_remote_encryptedFields", + // TODO(GODRIVER-3403): Support QE with Client.bulkWrite + "Support QE with Client.bulkWrite (GODRIVER-3403)": { + "TestUnifiedSpec/client-side-encryption/tests/unified/client-bulkWrite-qe.json/client_bulkWrite_QE_replaceOne", + "TestUnifiedSpec/client-side-encryption/tests/unified/client-bulkWrite-qe.json/client_bulkWrite_QE_with_multiple_replace_fails", }, // TODO(GODRIVER-3076): CSFLE/QE Support for more than 1 KMS provider per