Add include-vector-fields-by-default property to DataSource configuration

schemas/dab.draft.schema.json

@@ -61,6 +61,11 @@
               "maximum": 2147483647
             }
           }
+        },
+        "include-vector-fields-by-default": {
+          "type": "boolean",
+          "description": "When false (default), vector-type columns are omitted from results. Only applicable to mssql database type.",
+          "default": false
         }
       },
       "allOf": [
@@ -140,6 +145,23 @@
               }
             }
           }
+        },
+        {
+          "if": {
+            "properties": {
+              "include-vector-fields-by-default": {
+                "const": true
+              }
+            },
+            "required": ["include-vector-fields-by-default"]
+          },
+          "then": {
+            "properties": {
+              "database-type": {
+                "const": "mssql"
+              }
+            }
+          }
         }
       ],
       "required": ["database-type", "connection-string"]

src/Cli.Tests/ConfigureOptionsTests.cs

@@ -926,6 +926,100 @@ public void TestFailureWhenAddingSetSessionContextToMySQLDatabase()
             Assert.IsFalse(isSuccess);
         }
 
+        /// <summary>
+        /// Tests that running "dab configure --data-source.include-vector-fields-by-default true" on a MSSQL config
+        /// correctly updates the include-vector-fields-by-default property to true.
+        /// </summary>
+        [TestMethod]
+        public void TestAddIncludeVectorFieldsByDefaultForMSSQL()
+        {
+            // Arrange
+            SetupFileSystemWithInitialConfig(INITIAL_CONFIG);
+
+            // Act: Attempts to add include-vector-fields-by-default option
+            ConfigureOptions options = new(
+                dataSourceIncludeVectorFieldsByDefault: true,
+                config: TEST_RUNTIME_CONFIG_FILE
+            );
+            bool isSuccess = TryConfigureSettings(options, _runtimeConfigLoader!, _fileSystem!);
+
+            // Assert: Validate the include-vector-fields-by-default is added
+            Assert.IsTrue(isSuccess);
+            string updatedConfig = _fileSystem!.File.ReadAllText(TEST_RUNTIME_CONFIG_FILE);
+            Assert.IsTrue(RuntimeConfigLoader.TryParseConfig(updatedConfig, out RuntimeConfig? config));
+            Assert.IsNotNull(config.DataSource);
+            Assert.IsTrue(config.DataSource.IncludeVectorFieldsByDefault);
+        }
+
+        /// <summary>
+        /// Tests that running "dab configure --data-source.include-vector-fields-by-default false" on a MSSQL config
+        /// correctly updates the include-vector-fields-by-default property to false.
+        /// </summary>
+        [TestMethod]
+        public void TestSetIncludeVectorFieldsByDefaultToFalseForMSSQL()
+        {
+            // Arrange
+            SetupFileSystemWithInitialConfig(INITIAL_CONFIG);
+
+            // Act: Attempts to set include-vector-fields-by-default to false
+            ConfigureOptions options = new(
+                dataSourceIncludeVectorFieldsByDefault: false,
+                config: TEST_RUNTIME_CONFIG_FILE
+            );
+            bool isSuccess = TryConfigureSettings(options, _runtimeConfigLoader!, _fileSystem!);
+
+            // Assert: Validate the include-vector-fields-by-default is set to false
+            Assert.IsTrue(isSuccess);
+            string updatedConfig = _fileSystem!.File.ReadAllText(TEST_RUNTIME_CONFIG_FILE);
+            Assert.IsTrue(RuntimeConfigLoader.TryParseConfig(updatedConfig, out RuntimeConfig? config));
+            Assert.IsNotNull(config.DataSource);
+            Assert.IsFalse(config.DataSource.IncludeVectorFieldsByDefault);
+        }
+
+        /// <summary>
+        /// Tests that running "dab configure --data-source.include-vector-fields-by-default true" on a non-MSSQL config
+        /// (e.g., MySQL) fails with an error as this option is only applicable for MSSQL.
+        /// </summary>
+        [TestMethod]
+        public void TestFailureWhenAddingIncludeVectorFieldsByDefaultToMySQLDatabase()
+        {
+            // Arrange
+            SetupFileSystemWithInitialConfig(INITIAL_CONFIG);
+
+            // Act: Attempts to set include-vector-fields-by-default on MySQL database type
+            ConfigureOptions options = new(
+                dataSourceDatabaseType: "mysql",
+                dataSourceIncludeVectorFieldsByDefault: true,
+                config: TEST_RUNTIME_CONFIG_FILE
+            );
+            bool isSuccess = TryConfigureSettings(options, _runtimeConfigLoader!, _fileSystem!);
+
+            // Assert
+            Assert.IsFalse(isSuccess);
+        }
+
+        /// <summary>
+        /// Tests that running "dab configure --data-source.include-vector-fields-by-default true" on a PostgreSQL config
+        /// fails with an error as this option is only applicable for MSSQL.
+        /// </summary>
+        [TestMethod]
+        public void TestFailureWhenAddingIncludeVectorFieldsByDefaultToPostgreSQLDatabase()
+        {
+            // Arrange
+            SetupFileSystemWithInitialConfig(INITIAL_CONFIG);
+
+            // Act: Attempts to set include-vector-fields-by-default on PostgreSQL database type
+            ConfigureOptions options = new(
+                dataSourceDatabaseType: "postgresql",
+                dataSourceIncludeVectorFieldsByDefault: true,
+                config: TEST_RUNTIME_CONFIG_FILE
+            );
+            bool isSuccess = TryConfigureSettings(options, _runtimeConfigLoader!, _fileSystem!);
+
+            // Assert
+            Assert.IsFalse(isSuccess);
+        }
+
         /// <summary>
         /// Sets up the mock file system with an initial configuration file.
         /// This method adds a config file to the mock file system and verifies its existence.

src/Cli/Commands/ConfigureOptions.cs

@@ -28,6 +28,7 @@ public ConfigureOptions(
             string? dataSourceOptionsContainer = null,
             string? dataSourceOptionsSchema = null,
             bool? dataSourceOptionsSetSessionContext = null,
+            bool? dataSourceIncludeVectorFieldsByDefault = null,
             int? depthLimit = null,
             bool? runtimeGraphQLEnabled = null,
             string? runtimeGraphQLPath = null,
@@ -80,6 +81,7 @@ public ConfigureOptions(
             DataSourceOptionsContainer = dataSourceOptionsContainer;
             DataSourceOptionsSchema = dataSourceOptionsSchema;
             DataSourceOptionsSetSessionContext = dataSourceOptionsSetSessionContext;
+            DataSourceIncludeVectorFieldsByDefault = dataSourceIncludeVectorFieldsByDefault;
             // GraphQL
             DepthLimit = depthLimit;
             RuntimeGraphQLEnabled = runtimeGraphQLEnabled;
@@ -150,6 +152,9 @@ public ConfigureOptions(
         [Option("data-source.options.set-session-context", Required = false, HelpText = "Enable session context. Allowed values: true (default), false.")]
         public bool? DataSourceOptionsSetSessionContext { get; }
 
+        [Option("data-source.include-vector-fields-by-default", Required = false, HelpText = "Include vector-type columns in results by default. Only applicable to MSSQL. Default: false (boolean).")]
+        public bool? DataSourceIncludeVectorFieldsByDefault { get; }
+
         [Option("runtime.graphql.depth-limit", Required = false, HelpText = "Max allowed depth of the nested query. Allowed values: (0,2147483647] inclusive. Default is infinity. Use -1 to remove limit.")]
         public int? DepthLimit { get; }
 

src/Cli/ConfigGenerator.cs

@@ -643,6 +643,8 @@ private static bool TryUpdateConfiguredDataSourceOptions(
             DatabaseType dbType = runtimeConfig.DataSource.DatabaseType;
             string dataSourceConnectionString = runtimeConfig.DataSource.ConnectionString;
             DatasourceHealthCheckConfig? datasourceHealthCheckConfig = runtimeConfig.DataSource.Health;
+            bool includeVectorFieldsByDefault = runtimeConfig.DataSource.IncludeVectorFieldsByDefault;
+            bool userProvidedIncludeVectorFieldsByDefault = runtimeConfig.DataSource.UserProvidedIncludeVectorFieldsByDefault;
 
             if (options.DataSourceDatabaseType is not null)
             {
@@ -684,8 +686,24 @@ private static bool TryUpdateConfiguredDataSourceOptions(
                 dbOptions.Add(namingPolicy.ConvertName(nameof(MsSqlOptions.SetSessionContext)), options.DataSourceOptionsSetSessionContext.Value);
             }
 
+            if (options.DataSourceIncludeVectorFieldsByDefault is not null)
+            {
+                if (!DatabaseType.MSSQL.Equals(dbType))
+                {
+                    _logger.LogError("include-vector-fields-by-default option is only applicable for MSSQL database type.");
+                    return false;
+                }
+
+                includeVectorFieldsByDefault = options.DataSourceIncludeVectorFieldsByDefault.Value;
+                userProvidedIncludeVectorFieldsByDefault = true;
+                _logger.LogInformation("Updated RuntimeConfig with data-source.include-vector-fields-by-default as '{updatedValue}'", includeVectorFieldsByDefault);
+            }
+
             dbOptions = EnumerableUtilities.IsNullOrEmpty(dbOptions) ? null : dbOptions;
-            DataSource dataSource = new(dbType, dataSourceConnectionString, dbOptions, datasourceHealthCheckConfig);
+            DataSource dataSource = new(dbType, dataSourceConnectionString, dbOptions, datasourceHealthCheckConfig, includeVectorFieldsByDefault)
+            {
+                UserProvidedIncludeVectorFieldsByDefault = userProvidedIncludeVectorFieldsByDefault
+            };
             runtimeConfig = runtimeConfig with { DataSource = dataSource };
 
             return runtimeConfig != null;

src/Config/Converters/DataSourceConverterFactory.cs

@@ -51,12 +51,17 @@ public DataSourceConverter(DeserializationVariableReplacementSettings? replaceme
                 string connectionString = string.Empty;
                 DatasourceHealthCheckConfig? health = null;
                 Dictionary<string, object?>? datasourceOptions = null;
+                bool includeVectorFieldsByDefault = false;
+                bool userProvidedIncludeVectorFieldsByDefault = false;
 
                 while (reader.Read())
                 {
                     if (reader.TokenType is JsonTokenType.EndObject)
                     {
-                        return new DataSource(databaseType, connectionString, datasourceOptions, health);
+                        return new DataSource(databaseType, connectionString, datasourceOptions, health, includeVectorFieldsByDefault)
+                        {
+                            UserProvidedIncludeVectorFieldsByDefault = userProvidedIncludeVectorFieldsByDefault
+                        };
                     }
 
                     if (reader.TokenType is JsonTokenType.PropertyName)
@@ -136,6 +141,24 @@ public DataSourceConverter(DeserializationVariableReplacementSettings? replaceme
                                     datasourceOptions = optionsDict;
                                 }
 
+                                break;
+                            case "include-vector-fields-by-default":
+                                if (reader.TokenType is JsonTokenType.True or JsonTokenType.False)
+                                {
+                                    includeVectorFieldsByDefault = reader.GetBoolean();
+                                    userProvidedIncludeVectorFieldsByDefault = true;
+                                }
+                                else if (reader.TokenType is JsonTokenType.String)
+                                {
+                                    // Support environment variable replacement
+                                    string stringValue = reader.DeserializeString(_replacementSettings)!;
+                                    if (bool.TryParse(stringValue, out bool boolValue))
+                                    {
+                                        includeVectorFieldsByDefault = boolValue;
+                                        userProvidedIncludeVectorFieldsByDefault = true;
+                                    }
+                                }
+
                                 break;
                             default:
                                 throw new JsonException($"Unexpected property {propertyName} while deserializing DataSource.");
@@ -149,11 +172,37 @@ public DataSourceConverter(DeserializationVariableReplacementSettings? replaceme
 
         public override void Write(Utf8JsonWriter writer, DataSource value, JsonSerializerOptions options)
         {
-            // Remove the converter so we don't recurse.
-            JsonSerializerOptions innerOptions = new(options);
-            innerOptions.Converters.Remove(innerOptions.Converters.First(c => c is DataSourceConverterFactory));
+            writer.WriteStartObject();
+
+            // Always write required properties
+            writer.WritePropertyName("database-type");
+            JsonSerializer.Serialize(writer, value.DatabaseType, options);
+
+            writer.WritePropertyName("connection-string");
+            writer.WriteStringValue(value.ConnectionString);
+
+            // Write options if present
+            if (value.Options is not null && value.Options.Count > 0)
+            {
+                writer.WritePropertyName("options");
+                JsonSerializer.Serialize(writer, value.Options, options);
+            }
+
+            // Write health if present
+            if (value.Health is not null)
+            {
+                writer.WritePropertyName("health");
+                JsonSerializer.Serialize(writer, value.Health, options);
+            }
+
+            // Write include-vector-fields-by-default only if user provided it
+            if (value.UserProvidedIncludeVectorFieldsByDefault)
+            {
+                writer.WritePropertyName("include-vector-fields-by-default");
+                writer.WriteBooleanValue(value.IncludeVectorFieldsByDefault);
+            }
 
-            JsonSerializer.Serialize(writer, value, innerOptions);
+            writer.WriteEndObject();
         }
     }
 }

src/Config/DatabasePrimitives/DatabaseObject.cs

@@ -269,6 +269,13 @@ public class ColumnDefinition
     public bool IsReadOnly { get; set; }
     public object? DefaultValue { get; set; }
 
+    /// <summary>
+    /// Indicates whether this column is a vector type (SQL Server vector data type).
+    /// Vector columns can be large and are excluded from results by default when
+    /// include-vector-fields-by-default is set to false in the configuration.
+    /// </summary>
+    public bool IsVectorType { get; set; }
+
     public ColumnDefinition() { }
 
     public ColumnDefinition(Type systemType)

src/Config/ObjectModel/DataSource.cs

@@ -14,12 +14,23 @@ namespace Azure.DataApiBuilder.Config.ObjectModel;
 /// <param name="ConnectionString">Connection string to access the database.</param>
 /// <param name="Options">Custom options for the specific database. If there are no options, this could be null.</param>
 /// <param name="Health">Health check configuration for the datasource.</param>
+/// <param name="IncludeVectorFieldsByDefault">When false (default), vector-type columns are omitted from results. Only applicable to mssql.</param>
 public record DataSource(
     DatabaseType DatabaseType,
     string ConnectionString,
     Dictionary<string, object?>? Options = null,
-    DatasourceHealthCheckConfig? Health = null)
+    DatasourceHealthCheckConfig? Health = null,
+    bool IncludeVectorFieldsByDefault = false)
 {
+    /// <summary>
+    /// Flag which informs CLI and JSON serializer whether to write include-vector-fields-by-default
+    /// property and value to the runtime config file.
+    /// When user doesn't provide the property/value, which signals DAB to use the default (false),
+    /// the DAB CLI should not write the default value to a serialized config.
+    /// </summary>
+    [JsonIgnore(Condition = JsonIgnoreCondition.Always)]
+    public bool UserProvidedIncludeVectorFieldsByDefault { get; init; } = false;
+
     [JsonIgnore]
     public bool IsDatasourceHealthEnabled =>
         Health is null || Health.Enabled;