Feature/updates

.claude/settings.local.json

@@ -0,0 +1,17 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(cargo build:*)",
+      "Bash(cargo check:*)",
+      "Bash(where clang:*)",
+      "Read(//c/Program Files/LLVM/bin/**)",
+      "Bash(echo \"LIBCLANG_PATH=$LIBCLANG_PATH\")",
+      "Read(//c/Program Files/**)",
+      "Bash(winget list:*)",
+      "Bash(cargo tree:*)",
+      "Bash(where llvm-config:*)",
+      "Read(//c/msys64/mingw64/bin/**)",
+      "Read(//c/msys64/clang64/bin/**)"
+    ]
+  }
+}

README.md

@@ -68,6 +68,8 @@ examples/databricks/serverless dev \
 -e AWS_ACCOUNT_ID=${AWS_ACCOUNT_ID} \
 -e DATABRICKS_ACCOUNT_ID=${DATABRICKS_ACCOUNT_ID} \
 -e DATABRICKS_AWS_ACCOUNT_ID=${DATABRICKS_AWS_ACCOUNT_ID} \
+--show-queries \
+--log-level debug \
 --dry-run
 
 pgrep -f "stackql srv"

examples/databricks/serverless/resources/aws/iam/policy_statements/metastore_access_role/kms_decrypt.json

@@ -0,0 +1,11 @@
+{
+  "Effect": "Allow",
+  "Action": [
+    "kms:Decrypt",
+    "kms:Encrypt",
+    "kms:GenerateDataKey*"
+  ],
+  "Resource": [
+    "arn:aws:kms:<KMS-KEY>"
+  ]
+}

examples/databricks/serverless/resources/aws/iam/policy_statements/metastore_access_role/managed_file_events_list.json

@@ -0,0 +1,13 @@
+{
+  "Sid": "ManagedFileEventsListStatement",
+  "Effect": "Allow",
+  "Action": [
+    "sqs:ListQueues",
+    "sqs:ListQueueTags",
+    "sns:ListTopics"
+  ],
+  "Resource": [
+    "arn:aws:sqs:*:*:csms-*",
+    "arn:aws:sns:*:*:csms-*"
+  ]
+}

examples/databricks/serverless/resources/aws/iam/policy_statements/metastore_access_role/managed_file_events_setup.json

@@ -0,0 +1,30 @@
+{
+  "Sid": "ManagedFileEventsSetupStatement",
+  "Effect": "Allow",
+  "Action": [
+    "s3:GetBucketNotification",
+    "s3:PutBucketNotification",
+    "sns:ListSubscriptionsByTopic",
+    "sns:GetTopicAttributes",
+    "sns:SetTopicAttributes",
+    "sns:CreateTopic",
+    "sns:TagResource",
+    "sns:Publish",
+    "sns:Subscribe",
+    "sqs:CreateQueue",
+    "sqs:DeleteMessage",
+    "sqs:ReceiveMessage",
+    "sqs:SendMessage",
+    "sqs:GetQueueUrl",
+    "sqs:GetQueueAttributes",
+    "sqs:SetQueueAttributes",
+    "sqs:TagQueue",
+    "sqs:ChangeMessageVisibility",
+    "sqs:PurgeQueue"
+  ],
+  "Resource": [
+    "{{ aws_s3_metastore_bucket.arn }}",
+    "arn:aws:sqs:*:*:csms-*",
+    "arn:aws:sns:*:*:csms-*"
+  ]
+}

examples/databricks/serverless/resources/aws/iam/policy_statements/metastore_access_role/managed_file_events_teardown.json

@@ -0,0 +1,13 @@
+{
+  "Sid": "ManagedFileEventsTeardownStatement",
+  "Effect": "Allow",
+  "Action": [
+    "sns:Unsubscribe",
+    "sns:DeleteTopic",
+    "sqs:DeleteQueue"
+  ],
+  "Resource": [
+    "arn:aws:sqs:*:*:csms-*",
+    "arn:aws:sns:*:*:csms-*"
+  ]
+}

examples/databricks/serverless/resources/aws/iam/policy_statements/metastore_access_role/s3_access.json

@@ -0,0 +1,17 @@
+{
+  "Effect": "Allow",
+  "Action": [
+    "s3:GetObject",
+    "s3:PutObject",
+    "s3:DeleteObject",
+    "s3:ListBucket",
+    "s3:GetBucketLocation",
+    "s3:ListBucketMultipartUploads",
+    "s3:ListMultipartUploadParts",
+    "s3:AbortMultipartUpload"
+  ],
+  "Resource": [
+    "{{ aws_s3_metastore_bucket.arn }}/*",
+    "{{ aws_s3_metastore_bucket.arn }}"
+  ]
+}

examples/databricks/serverless/resources/aws/iam/policy_statements/metastore_access_role/sts_assume_role.json

@@ -0,0 +1,9 @@
+{
+  "Effect": "Allow",
+  "Action": [
+    "sts:AssumeRole"
+  ],
+  "Resource": [
+    "arn:aws:iam::{{ databricks_aws_account_id }}:role/{{ stack_name }}-{{ stack_env }}-metastore-role"
+  ]
+}

examples/databricks/serverless/resources/aws/s3/bucket_policies.iql

@@ -0,0 +1,44 @@
+/*+ exists */
+SELECT count(*) as count
+FROM awscc.s3.bucket_policies
+WHERE region = '{{ region }}' AND
+Identifier = '{{ bucket_name }}'
+;
+
+/*+ create */
+INSERT INTO awscc.s3.bucket_policies (
+  Bucket,
+  PolicyDocument,
+  region
+)
+SELECT
+  '{{ bucket_name }}',
+  '{{ policy_document }}',
+  '{{ region }}';
+
+/*+ update */
+UPDATE awscc.s3.bucket_policies
+SET PatchDocument = string('{{ {
+    "PolicyDocument": policy_document
+} | generate_patch_document }}')
+WHERE
+  region = '{{ region }}' AND
+  Identifier = '{{ bucket_name }}';
+
+/*+ statecheck, retries=5, retry_delay=10 */
+SELECT 
+count(*) as count
+FROM (
+    SELECT 
+        AWS_POLICY_EQUAL(policy_document, '{{ policy_document }}') as test_policy_document
+    FROM awscc.s3.bucket_policies 
+    WHERE region = '{{ region }}' 
+    AND Identifier = '{{ bucket_name }}'
+)t
+WHERE test_policy_document = 1;
+
+/*+ delete */
+DELETE FROM awscc.s3.bucket_policies
+WHERE
+  Identifier = '{{ bucket_name }}' AND
+  region = '{{ region }}';

examples/databricks/serverless/resources/aws/s3/buckets.iql

@@ -1,7 +1,7 @@
 /*+ exists */
 SELECT count(*) as count
 FROM awscc.s3.buckets
-WHERE region = 'us-east-1' AND
+WHERE region = '{{ region }}' AND
 Identifier = '{{ bucket_name }}'
 ;
 
@@ -36,29 +36,30 @@ SET PatchDocument = string('{{ {
 WHERE region = '{{ region }}'
 AND Identifier = '{{ bucket_name }}';
 
-/*+ statecheck, retries=3, retry_delay=2 */
-SELECT count(*) as count
-FROM awscc.s3.buckets
-WHERE 
-region = 'us-east-1' AND
-Identifier = '{{ bucket_name }}' AND
-JSON_EQUAL(bucket_encryption, '{{ bucket_encryption }}') AND
-JSON_EQUAL(ownership_controls, '{{ ownership_controls }}') AND
-JSON_EQUAL(public_access_block_configuration, '{{ public_access_block_configuration }}') AND
-JSON_EQUAL(tags, '{{ tags }}') AND
-JSON_EQUAL(versioning_configuration, '{{ versioning_configuration }}')
-;
-
-
-/*+ exports */
-SELECT
-bucket_name,
-arn
-FROM awscc.s3.buckets
-WHERE region = 'us-east-1' AND
-Identifier = '{{ bucket_name }}';
+/*+ exports, retries=3, retry_delay=5 */
+SELECT 
+arn,
+bucket_name
+FROM (
+    SELECT 
+        arn,
+        bucket_name,
+        JSON_EQUAL(ownership_controls, '{{ ownership_controls }}') as test_ownership_controls,
+        JSON_EQUAL(bucket_encryption, '{{ bucket_encryption }}') as test_bucket_encryption,
+        JSON_EQUAL(public_access_block_configuration, '{{ public_access_block_configuration }}') as test_public_access_block_configuration,
+        JSON_EQUAL(versioning_configuration, '{{ versioning_configuration }}') as test_versioning_configuration,
+        AWS_POLICY_EQUAL(tags, '{{ tags }}') as test_tags
+    FROM awscc.s3.buckets 
+    WHERE region = '{{ region }}' 
+    AND Identifier = '{{ bucket_name }}'
+)t
+WHERE test_ownership_controls = 1
+AND test_bucket_encryption = 1
+AND test_public_access_block_configuration = 1
+AND test_versioning_configuration = 1
+AND test_tags = 1;
 
 /*+ delete */
 DELETE FROM awscc.s3.buckets
 WHERE Identifier = '{{ bucket_name }}'
-AND region = 'us-east-1';
+AND region = '{{ region }}';

examples/databricks/serverless/resources/databricks_account/account_groups.iql

@@ -0,0 +1,83 @@
+/*+ exists */
+SELECT count(*) as count
+FROM databricks_account.iam.account_groups
+WHERE account_id = '{{ account_id }}'
+AND filter = 'displayName Eq "{{ displayName }}"'
+;
+
+/*+ create */
+INSERT INTO databricks_account.iam.account_groups (
+display_name,
+external_id,
+id,
+members,
+meta,
+roles,
+account_id
+)
+SELECT 
+'{{ display_name }}',
+'{{ external_id }}',
+'{{ id }}',
+'{{ members }}',
+'{{ meta }}',
+'{{ roles }}',
+'{{ account_id }}'
+RETURNING
+id,
+account_id,
+displayName,
+externalId,
+members,
+meta,
+roles
+;
+
+/*+ update */
+UPDATE databricks_account.iam.account_groups
+SET 
+operations = '{{ operations }}',
+schemas = '{{ schemas }}'
+WHERE 
+account_id = '{{ account_id }}' --required
+AND id = '{{ id }}' --required;
+
+/*+ statecheck, retries=5, retry_delay=10 */
+SELECT count(*) as count
+FROM databricks_account.iam.account_groups
+WHERE 
+id = '{{ id }}' AND
+members = '{{ members }}' AND
+meta = '{{ meta }}' AND
+roles = '{{ roles }}' AND
+account_id = '{{ account_id }}' -- required
+AND attributes = '{{ attributes }}'
+AND count = '{{ count }}'
+AND excluded_attributes = '{{ excluded_attributes }}'
+AND filter = '{{ filter }}'
+AND sort_by = '{{ sort_by }}'
+AND sort_order = '{{ sort_order }}'
+AND start_index = '{{ start_index }}'
+;
+
+/*+ exports */
+SELECT id,
+members,
+meta,
+roles
+FROM databricks_account.iam.account_groups
+WHERE account_id = '{{ account_id }}' -- required
+AND attributes = '{{ attributes }}'
+AND count = '{{ count }}'
+AND excluded_attributes = '{{ excluded_attributes }}'
+AND filter = '{{ filter }}'
+AND sort_by = '{{ sort_by }}'
+AND sort_order = '{{ sort_order }}'
+AND start_index = '{{ start_index }}'
+;
+
+/*+ delete */
+DELETE FROM databricks_account.iam.account_groups
+WHERE account_id = '{{ account_id }}' --required
+AND id = '{{ id }}' --required
+;

examples/databricks/serverless/resources/databricks_account/storage.iql

@@ -0,0 +1,30 @@
+/*+ exists */
+SELECT count(*) as count
+FROM databricks_account.provisioning.storage
+WHERE account_id = '{{ account_id }}'
+AND storage_configuration_name = '{{ storage_configuration_name }}'
+;
+
+/*+ create */
+INSERT INTO databricks_account.provisioning.storage (
+storage_configuration_name,
+root_bucket_info,
+account_id
+)
+SELECT 
+'{{ storage_configuration_name }}',
+'{{ root_bucket_info }}',
+'{{ account_id }}';
+
+/*+ exports, retries=5, retry_delay=10 */
+SELECT storage_configuration_id
+FROM databricks_account.provisioning.storage
+WHERE account_id = '{{ account_id }}'
+AND storage_configuration_name = '{{ storage_configuration_name }}'
+AND JSON_EXTRACT(root_bucket_info, '$.bucket_name') = '{{ root_bucket_info | from_json | get(key="bucket_name") }}';
+
+/*+ delete */
+DELETE FROM databricks_account.provisioning.storage
+WHERE account_id = '{{ account_id }}'
+AND storage_configuration_id = '{{ storage_configuration_id }}'
+;