direct: Generate resources config based on OpenAPI field behaviours

.gitignore

@@ -54,3 +54,6 @@ dist/
 # Go workspace file
 go.work
 go.work.sum
+
+# Fetched based on recorded hash
+.codegen/openapi.json

Makefile

@@ -170,6 +170,21 @@ generate:
 	@echo "Generating CLI code..."
 	$(GENKIT_BINARY) update-sdk
 
+.codegen/openapi.json: .codegen/_openapi_sha
+	wget -O $@.tmp "https://openapi.dev.databricks.com/$$(cat $<)/specs/all-internal.json" && mv $@.tmp $@ && touch $@
+
+generate-direct: generate-direct-apitypes generate-direct-resources
+generate-direct-apitypes: bundle/direct/dresources/apitypes.generated.yml
+generate-direct-resources: bundle/direct/dresources/resources.generated.yml
+generate-direct-clean:
+	rm -f bundle/direct/dresources/apitypes.generated.yml bundle/direct/dresources/resources.generated.yml
+.PHONY: generate-direct generate-direct-apitypes generate-direct-resources generate-direct-clean
+
+bundle/direct/dresources/apitypes.generated.yml: ./bundle/direct/tools/generate_apitypes.py .codegen/openapi.json acceptance/bundle/refschema/out.fields.txt
+	python3 $^ > $@
+
+bundle/direct/dresources/resources.generated.yml: ./bundle/direct/tools/generate_resources.py .codegen/openapi.json bundle/direct/dresources/apitypes.generated.yml acceptance/bundle/refschema/out.fields.txt
+	python3 $^ > $@
 
 .PHONY: lint lintfull tidy lintcheck fmt fmtfull test test-unit test-acc test-slow test-slow-unit test-slow-acc cover showcover build snapshot snapshot-release schema integration integration-short acc-cover acc-showcover docs ws wsfix links checks test-update test-update-templates generate-out-test-toml test-update-aws test-update-all generate-validation
 

bundle/deployplan/plan.go

@@ -102,6 +102,7 @@ const (
 	ReasonAlias             = "alias"
 	ReasonRemoteAlreadySet  = "remote_already_set"
 	ReasonBuiltinRule       = "builtin_rule"
+	ReasonAPISchema         = "api_schema"
 	ReasonEmptySlice        = "empty_slice"
 	ReasonEmptyMap          = "empty_map"
 	ReasonEmptyStruct       = "empty_struct"

bundle/direct/bundle_plan.go

@@ -360,6 +360,7 @@ func prepareChanges(ctx context.Context, adapter *dresources.Adapter, localDiff,
 
 func addPerFieldActions(ctx context.Context, adapter *dresources.Adapter, changes deployplan.Changes, remoteState any) error {
 	cfg := adapter.ResourceConfig()
+	generatedCfg := adapter.GeneratedResourceConfig()
 
 	for pathString, ch := range changes {
 		path, err := structpath.Parse(pathString)
@@ -385,6 +386,9 @@ func addPerFieldActions(ctx context.Context, adapter *dresources.Adapter, change
 		} else if shouldSkip(cfg, path, ch) {
 			ch.Action = deployplan.Skip
 			ch.Reason = deployplan.ReasonBuiltinRule
+		} else if shouldSkip(generatedCfg, path, ch) {
+			ch.Action = deployplan.Skip
+			ch.Reason = deployplan.ReasonAPISchema
 		} else if ch.New == nil && ch.Old == nil && ch.Remote != nil && path.IsDotString() {
 			// The field was not set by us, but comes from the remote state.
 			// This could either be server-side default or a policy.
@@ -395,6 +399,9 @@ func addPerFieldActions(ctx context.Context, adapter *dresources.Adapter, change
 		} else if action := shouldUpdateOrRecreate(cfg, path); action != deployplan.Undefined {
 			ch.Action = action
 			ch.Reason = deployplan.ReasonBuiltinRule
+		} else if action := shouldUpdateOrRecreate(generatedCfg, path); action != deployplan.Undefined {
+			ch.Action = action
+			ch.Reason = deployplan.ReasonAPISchema
 		} else {
 			ch.Action = deployplan.Update
 		}

bundle/direct/dresources/adapter.go

@@ -94,8 +94,9 @@ type Adapter struct {
 	overrideChangeDesc *calladapt.BoundCaller
 	doResize           *calladapt.BoundCaller
 
-	resourceConfig *ResourceLifecycleConfig
-	keyedSlices    map[string]any
+	resourceConfig          *ResourceLifecycleConfig
+	generatedResourceConfig *ResourceLifecycleConfig
+	keyedSlices             map[string]any
 }
 
 func NewAdapter(typedNil any, resourceType string, client *databricks.WorkspaceClient) (*Adapter, error) {
@@ -112,19 +113,20 @@ func NewAdapter(typedNil any, resourceType string, client *databricks.WorkspaceC
 	}
 	impl := outs[0]
 	adapter := &Adapter{
-		prepareState:       nil,
-		remapState:         nil,
-		doRefresh:          nil,
-		doDelete:           nil,
-		doCreate:           nil,
-		doUpdate:           nil,
-		doUpdateWithID:     nil,
-		doResize:           nil,
-		waitAfterCreate:    nil,
-		waitAfterUpdate:    nil,
-		overrideChangeDesc: nil,
-		resourceConfig:     GetResourceConfig(resourceType),
-		keyedSlices:        nil,
+		prepareState:            nil,
+		remapState:              nil,
+		doRefresh:               nil,
+		doDelete:                nil,
+		doCreate:                nil,
+		doUpdate:                nil,
+		doUpdateWithID:          nil,
+		doResize:                nil,
+		waitAfterCreate:         nil,
+		waitAfterUpdate:         nil,
+		overrideChangeDesc:      nil,
+		resourceConfig:          GetResourceConfig(resourceType),
+		generatedResourceConfig: GetGeneratedResourceConfig(resourceType),
+		keyedSlices:             nil,
 	}
 
 	err = adapter.initMethods(impl)
@@ -355,6 +357,10 @@ func (a *Adapter) ResourceConfig() *ResourceLifecycleConfig {
 	return a.resourceConfig
 }
 
+func (a *Adapter) GeneratedResourceConfig() *ResourceLifecycleConfig {
+	return a.generatedResourceConfig
+}
+
 func (a *Adapter) IsFieldInRecreateOnChanges(path *structpath.PathNode) bool {
 	for _, p := range a.resourceConfig.RecreateOnChanges {
 		if path.HasPrefix(p) {

bundle/direct/dresources/all_test.go

@@ -659,51 +659,54 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W
 	}
 }
 
-// validateFields uses structwalk to generate all valid field paths and checks membership.
-func validateFields(t *testing.T, configType reflect.Type, fields map[string]deployplan.ActionType) {
-	validPaths := make(map[string]struct{})
-
-	err := structwalk.WalkType(configType, func(path *structpath.PathNode, typ reflect.Type, field *reflect.StructField) bool {
-		validPaths[path.String()] = struct{}{}
-		return true // continue walking
-	})
-	require.NoError(t, err)
+// TestResourceConfig validates that all field patterns in resource config
+// exist in the corresponding StateType for each resource.
+func TestResourceConfig(t *testing.T) {
+	for resourceType, resource := range SupportedResources {
+		adapter, err := NewAdapter(resource, resourceType, nil)
+		require.NoError(t, err)
 
-	for fieldPath := range fields {
-		if _, exists := validPaths[fieldPath]; !exists {
-			t.Errorf("invalid field '%s' for %s", fieldPath, configType)
+		cfg := adapter.ResourceConfig()
+		if cfg == nil {
+			continue
 		}
+
+		t.Run(resourceType, func(t *testing.T) {
+			validateResourceConfig(t, adapter.StateType(), cfg)
+		})
 	}
 }
 
-// TestResourceConfig validates that all field patterns in resource config
+// TestGeneratedResourceConfig validates that all field patterns in generated resource config
 // exist in the corresponding StateType for each resource.
-func TestResourceConfig(t *testing.T) {
+func TestGeneratedResourceConfig(t *testing.T) {
 	for resourceType, resource := range SupportedResources {
 		adapter, err := NewAdapter(resource, resourceType, nil)
 		require.NoError(t, err)
 
-		cfg := adapter.ResourceConfig()
+		cfg := adapter.GeneratedResourceConfig()
 		if cfg == nil {
 			continue
 		}
 
 		t.Run(resourceType, func(t *testing.T) {
-			fieldMap := make(map[string]deployplan.ActionType)
-			for _, p := range cfg.RecreateOnChanges {
-				fieldMap[p.String()] = deployplan.Recreate
-			}
-			for _, p := range cfg.UpdateIDOnChanges {
-				fieldMap[p.String()] = deployplan.UpdateWithID
-			}
-			for _, p := range cfg.IgnoreRemoteChanges {
-				fieldMap[p.String()] = deployplan.Skip
-			}
-			validateFields(t, adapter.StateType(), fieldMap)
+			validateResourceConfig(t, adapter.StateType(), cfg)
 		})
 	}
 }
 
+func validateResourceConfig(t *testing.T, stateType reflect.Type, cfg *ResourceLifecycleConfig) {
+	for _, p := range cfg.RecreateOnChanges {
+		assert.NoError(t, structaccess.Validate(stateType, p), "RecreateOnChanges: %s", p)
+	}
+	for _, p := range cfg.UpdateIDOnChanges {
+		assert.NoError(t, structaccess.Validate(stateType, p), "UpdateIDOnChanges: %s", p)
+	}
+	for _, p := range cfg.IgnoreRemoteChanges {
+		assert.NoError(t, structaccess.Validate(stateType, p), "IgnoreRemoteChanges: %s", p)
+	}
+}
+
 func setupTestServerClient(t *testing.T) (*testserver.Server, *databricks.WorkspaceClient) {
 	server := testserver.New(t)
 	testserver.AddDefaultHandlers(server)

bundle/direct/dresources/apitypes.generated.yml

@@ -0,0 +1,37 @@
+# Generated, do not edit. Override via apitypes.yml
+
+alerts: sql.AlertV2
+
+apps: apps.App
+
+clusters: compute.ClusterSpec
+
+dashboards: dashboards.Dashboard
+
+database_catalogs: database.DatabaseCatalog
+
+database_instances: database.DatabaseInstance
+
+experiments: ml.CreateExperiment
+
+jobs: jobs.JobSettings
+
+model_serving_endpoints: serving.CreateServingEndpoint
+
+models: ml.CreateModelRequest
+
+pipelines: pipelines.CreatePipeline
+
+quality_monitors: catalog.CreateMonitor
+
+registered_models: catalog.RegisteredModelInfo
+
+schemas: catalog.CreateSchema
+
+secret_scopes: workspace.CreateScope
+
+sql_warehouses: sql.EditWarehouseRequest
+
+synced_database_tables: database.SyncedDatabaseTable
+
+volumes: catalog.CreateVolumeRequestContent

bundle/direct/dresources/config.go

@@ -31,9 +31,14 @@ type Config struct {
 //go:embed resources.yml
 var resourcesYAML []byte
 
+//go:embed resources.generated.yml
+var resourcesGeneratedYAML []byte
+
 var (
-	configOnce   sync.Once
-	globalConfig *Config
+	configOnce          sync.Once
+	globalConfig        *Config
+	generatedConfigOnce sync.Once
+	generatedConfig     *Config
 )
 
 // MustLoadConfig loads and parses the embedded resources.yml configuration.
@@ -51,6 +56,21 @@ func MustLoadConfig() *Config {
 	return globalConfig
 }
 
+// MustLoadGeneratedConfig loads and parses the embedded resources.generated.yml configuration.
+// The config is loaded once and cached for subsequent calls.
+// Panics if the embedded YAML is invalid.
+func MustLoadGeneratedConfig() *Config {
+	generatedConfigOnce.Do(func() {
+		generatedConfig = &Config{
+			Resources: nil,
+		}
+		if err := yaml.Unmarshal(resourcesGeneratedYAML, generatedConfig); err != nil {
+			panic(err)
+		}
+	})
+	return generatedConfig
+}
+
 // GetResourceConfig returns the lifecycle config for a given resource type.
 // Returns nil if the resource type has no configuration.
 func GetResourceConfig(resourceType string) *ResourceLifecycleConfig {
@@ -60,3 +80,13 @@ func GetResourceConfig(resourceType string) *ResourceLifecycleConfig {
 	}
 	return nil
 }
+
+// GetGeneratedResourceConfig returns the generated lifecycle config for a given resource type.
+// Returns nil if the resource type has no configuration.
+func GetGeneratedResourceConfig(resourceType string) *ResourceLifecycleConfig {
+	cfg := MustLoadGeneratedConfig()
+	if rc, ok := cfg.Resources[resourceType]; ok {
+		return &rc
+	}
+	return nil
+}