From 5546ad8513a3dbc84a2c961dd5cc5943e5a26f36 Mon Sep 17 00:00:00 2001 From: mengw15 <125719918+mengw15@users.noreply.github.com> Date: Mon, 9 Mar 2026 03:53:27 -0700 Subject: [PATCH 1/9] lakekeeper single node deployment --- bin/single-node/.env | 14 ++- bin/single-node/docker-compose.yml | 140 +++++++++++++++++++++++++++++ 2 files changed, 153 insertions(+), 1 deletion(-) diff --git a/bin/single-node/.env b/bin/single-node/.env index 2c949de0fbe..094ce25416a 100644 --- a/bin/single-node/.env +++ b/bin/single-node/.env @@ -58,6 +58,18 @@ STORAGE_ICEBERG_CATALOG_POSTGRES_URI_WITHOUT_SCHEME=texera-postgres:5432/texera_ STORAGE_ICEBERG_CATALOG_POSTGRES_USERNAME=texera STORAGE_ICEBERG_CATALOG_POSTGRES_PASSWORD=password +LAKEKEEPER__PG_DATABASE_URL_READ=postgres://texera:password@texera-postgres:5432/texera_lakekeeper +LAKEKEEPER__PG_DATABASE_URL_WRITE=postgres://texera:password@texera-postgres:5432/texera_lakekeeper +LAKEKEEPER__PG_ENCRYPTION_KEY=texera_key +LAKEKEEPER_BASE_URI=http://texera-lakekeeper:8181 +STORAGE_ICEBERG_CATALOG_REST_WAREHOUSE_NAME=texera +STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET=texera-iceberg +STORAGE_S3_AUTH_USERNAME=texera_minio +STORAGE_S3_AUTH_PASSWORD=password +STORAGE_ICEBERG_CATALOG_REST_REGION=us-west-2 + +STORAGE_ICEBERG_CATALOG_REST_URI=http://texera-lakekeeper:8181/catalog +STORAGE_ICEBERG_CATALOG_TYPE=rest # Admin credentials for Texera (used for login and example data loading) USER_SYS_ADMIN_USERNAME=texera -USER_SYS_ADMIN_PASSWORD=texera \ No newline at end of file +USER_SYS_ADMIN_PASSWORD=texera diff --git a/bin/single-node/docker-compose.yml b/bin/single-node/docker-compose.yml index 8c63d0d3652..0fb5ded49ed 100644 --- a/bin/single-node/docker-compose.yml +++ b/bin/single-node/docker-compose.yml @@ -75,6 +75,142 @@ services: timeout: 5s retries: 10 + # Lakekeeper migration init container + # This runs once to migrate the database before the lakekeeper server starts + lakekeeper-migrate: + image: vakamo/lakekeeper:v0.11.0 + container_name: texera-lakekeeper-migrate + depends_on: + postgres: + condition: service_healthy + env_file: + - .env + restart: "no" + entrypoint: ["/home/nonroot/lakekeeper"] + command: ["migrate"] + + # Lakekeeper is the Iceberg REST catalog service + lakekeeper: + image: vakamo/lakekeeper:v0.11.0 + container_name: texera-lakekeeper + restart: always + depends_on: + postgres: + condition: service_healthy + minio: + condition: service_started + lakekeeper-migrate: + condition: service_completed_successfully + env_file: + - .env + entrypoint: ["/home/nonroot/lakekeeper"] + command: ["serve"] + ports: + - "8181:8181" + healthcheck: + test: ["CMD", "/home/nonroot/lakekeeper", "healthcheck"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 10s + + lakekeeper-init: + image: alpine:3.19 + container_name: texera-lakekeeper-init + depends_on: + lakekeeper: + condition: service_healthy + minio: + condition: service_started + env_file: + - .env + restart: "no" + entrypoint: [ "/bin/sh", "-c" ] + command: + - | + set -e + + echo "Installing dependencies..." + apk add --no-cache curl ca-certificates + + echo "Installing MinIO Client..." + wget -q https://dl.min.io/client/mc/release/linux-amd64/mc -O /usr/local/bin/mc + chmod +x /usr/local/bin/mc + + check_status() { + if [ "$$1" -ge 200 ] && [ "$$1" -lt 300 ]; then + echo "Created $$2 successfully (HTTP $$1)." + elif [ "$$1" -eq 409 ]; then + echo "$$2 already exists (HTTP 409). Treating as success." + else + echo "Failed to create $$2. HTTP Code: $$1" + echo "ERROR RESPONSE:" + if [ -f /tmp/response.txt ]; then cat /tmp/response.txt; fi + echo "" + exit 1 + fi + } + + echo "Step 1: Initializing MinIO bucket '$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET'..." + mc alias set minio "$$STORAGE_S3_ENDPOINT" "$$STORAGE_S3_AUTH_USERNAME" "$$STORAGE_S3_AUTH_PASSWORD" || true + if mc ls minio/$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET > /dev/null 2>&1; then + echo "MinIO bucket '$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET' already exists." + else + mc mb minio/$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET || { + echo "Failed to create MinIO bucket '$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET'" + exit 1 + } + echo "MinIO bucket '$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET' created successfully." + fi + + + echo "Step 2: Initializing Default Project..." + PROJECT_PAYLOAD='{"project-id": "00000000-0000-0000-0000-000000000000", "project-name": "default"}' + + PROJECT_CODE=$$(curl -s -o /tmp/response.txt -w "%{http_code}" \ + -X POST \ + -H "Content-Type: application/json" \ + -d "$$PROJECT_PAYLOAD" \ + "$$LAKEKEEPER_BASE_URI/management/v1/project" || echo "000") + + check_status "$$PROJECT_CODE" "Default Project" + + + echo "Step 3: Initializing Warehouse '$$STORAGE_ICEBERG_CATALOG_REST_WAREHOUSE_NAME'..." + CREATE_PAYLOAD=$$(cat < Date: Tue, 14 Apr 2026 16:23:42 -0700 Subject: [PATCH 2/9] resolve comments --- bin/single-node/.env | 52 ++++++++++++++++++------------ bin/single-node/docker-compose.yml | 7 +++- 2 files changed, 37 insertions(+), 22 deletions(-) diff --git a/bin/single-node/.env b/bin/single-node/.env index 094ce25416a..dafec10cccc 100644 --- a/bin/single-node/.env +++ b/bin/single-node/.env @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +# Public host and ports exposed by the deployment TEXERA_HOST=http://localhost TEXERA_PORT=8080 MINIO_PORT=9000 @@ -27,49 +28,58 @@ TEXERA_SERVICE_LOG_LEVEL=INFO IMAGE_REGISTRY=ghcr.io/apache IMAGE_TAG=latest +# Admin credentials for Texera +USER_SYS_ADMIN_USERNAME=texera +USER_SYS_ADMIN_PASSWORD=texera + +# Postgres root credentials POSTGRES_USER=texera POSTGRES_PASSWORD=password -MINIO_ROOT_USER=texera_minio -MINIO_ROOT_PASSWORD=password +# S3 (MinIO) credentials +STORAGE_S3_AUTH_USERNAME=texera_minio +STORAGE_S3_AUTH_PASSWORD=password +# LakeFS server configuration LAKEFS_INSTALLATION_USER_NAME=texera-admin LAKEFS_INSTALLATION_ACCESS_KEY_ID=AKIAIOSFOLKFSSAMPLES LAKEFS_INSTALLATION_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY LAKEFS_BLOCKSTORE_TYPE=s3 LAKEFS_BLOCKSTORE_S3_FORCE_PATH_STYLE=true LAKEFS_BLOCKSTORE_S3_ENDPOINT=http://texera-minio:9000 -LAKEFS_BLOCKSTORE_S3_CREDENTIALS_ACCESS_KEY_ID=texera_minio -LAKEFS_BLOCKSTORE_S3_CREDENTIALS_SECRET_ACCESS_KEY=password LAKEFS_AUTH_ENCRYPT_SECRET_KEY=random_string_for_lakefs LAKEFS_LOGGING_LEVEL=INFO LAKEFS_STATS_ENABLED=1 LAKEFS_DATABASE_TYPE=postgres LAKEFS_DATABASE_POSTGRES_CONNECTION_STRING=postgres://texera:password@texera-postgres:5432/texera_lakefs?sslmode=disable +# Lakekeeper server configuration +LAKEKEEPER__PG_DATABASE_URL_READ=postgres://texera:password@texera-postgres:5432/texera_lakekeeper +LAKEKEEPER__PG_DATABASE_URL_WRITE=postgres://texera:password@texera-postgres:5432/texera_lakekeeper +LAKEKEEPER__PG_ENCRYPTION_KEY=texera_key +LAKEKEEPER_BASE_URI=http://texera-lakekeeper:8181 + +# Texera storage endpoints STORAGE_S3_ENDPOINT=http://texera-minio:9000 +STORAGE_S3_REGION=us-west-2 STORAGE_LAKEFS_ENDPOINT=http://texera-lakefs:8000/api/v1 STORAGE_JDBC_URL=jdbc:postgresql://texera-postgres:5432/texera_db?currentSchema=texera_db,public STORAGE_JDBC_USERNAME=texera STORAGE_JDBC_PASSWORD=password -FILE_SERVICE_GET_PRESIGNED_URL_ENDPOINT=http://file-service:9092/api/dataset/presign-download -FILE_SERVICE_UPLOAD_ONE_FILE_TO_DATASET_ENDPOINT=http://file-service:9092/api/dataset/did/upload -STORAGE_ICEBERG_CATALOG_POSTGRES_URI_WITHOUT_SCHEME=texera-postgres:5432/texera_iceberg_catalog -STORAGE_ICEBERG_CATALOG_POSTGRES_USERNAME=texera -STORAGE_ICEBERG_CATALOG_POSTGRES_PASSWORD=password -LAKEKEEPER__PG_DATABASE_URL_READ=postgres://texera:password@texera-postgres:5432/texera_lakekeeper -LAKEKEEPER__PG_DATABASE_URL_WRITE=postgres://texera:password@texera-postgres:5432/texera_lakekeeper -LAKEKEEPER__PG_ENCRYPTION_KEY=texera_key -LAKEKEEPER_BASE_URI=http://texera-lakekeeper:8181 +# Iceberg catalog selector (valid values: rest, postgres) +STORAGE_ICEBERG_CATALOG_TYPE=rest + +# Iceberg REST catalog client configuration +STORAGE_ICEBERG_CATALOG_REST_URI=http://texera-lakekeeper:8181/catalog STORAGE_ICEBERG_CATALOG_REST_WAREHOUSE_NAME=texera STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET=texera-iceberg -STORAGE_S3_AUTH_USERNAME=texera_minio -STORAGE_S3_AUTH_PASSWORD=password -STORAGE_ICEBERG_CATALOG_REST_REGION=us-west-2 -STORAGE_ICEBERG_CATALOG_REST_URI=http://texera-lakekeeper:8181/catalog -STORAGE_ICEBERG_CATALOG_TYPE=rest -# Admin credentials for Texera (used for login and example data loading) -USER_SYS_ADMIN_USERNAME=texera -USER_SYS_ADMIN_PASSWORD=texera +# Postgres-backed Iceberg catalog +STORAGE_ICEBERG_CATALOG_POSTGRES_URI_WITHOUT_SCHEME=texera-postgres:5432/texera_iceberg_catalog +STORAGE_ICEBERG_CATALOG_POSTGRES_USERNAME=texera +STORAGE_ICEBERG_CATALOG_POSTGRES_PASSWORD=password + +# File service endpoints +FILE_SERVICE_GET_PRESIGNED_URL_ENDPOINT=http://file-service:9092/api/dataset/presign-download +FILE_SERVICE_UPLOAD_ONE_FILE_TO_DATASET_ENDPOINT=http://file-service:9092/api/dataset/did/upload diff --git a/bin/single-node/docker-compose.yml b/bin/single-node/docker-compose.yml index 0fb5ded49ed..5d4b7cc91a6 100644 --- a/bin/single-node/docker-compose.yml +++ b/bin/single-node/docker-compose.yml @@ -26,6 +26,9 @@ services: - "${MINIO_PORT:-9000}:9000" env_file: - .env + environment: + - MINIO_ROOT_USER=${STORAGE_S3_AUTH_USERNAME} + - MINIO_ROOT_PASSWORD=${STORAGE_S3_AUTH_PASSWORD} volumes: - minio_data:/data command: server --console-address ":9001" /data @@ -63,6 +66,8 @@ services: environment: # This port also need to be changed if the port of MinIO service is changed - LAKEFS_BLOCKSTORE_S3_PRE_SIGNED_ENDPOINT=${TEXERA_HOST}:${MINIO_PORT:-9000} + - LAKEFS_BLOCKSTORE_S3_CREDENTIALS_ACCESS_KEY_ID=${STORAGE_S3_AUTH_USERNAME} + - LAKEFS_BLOCKSTORE_S3_CREDENTIALS_SECRET_ACCESS_KEY=${STORAGE_S3_AUTH_PASSWORD} entrypoint: ["/bin/sh", "-c"] command: - | @@ -184,7 +189,7 @@ services: "storage-profile": { "type": "s3", "bucket": "$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET", - "region": "$$STORAGE_ICEBERG_CATALOG_REST_REGION", + "region": "$$STORAGE_S3_REGION", "endpoint": "$$STORAGE_S3_ENDPOINT", "flavor": "s3-compat", "path-style-access": true, From c44c038a21cd8678d612aece1e3703150bb058b1 Mon Sep 17 00:00:00 2001 From: mengw15 <125719918+mengw15@users.noreply.github.com> Date: Tue, 14 Apr 2026 16:42:34 -0700 Subject: [PATCH 3/9] resolve comments --- bin/single-node/docker-compose.yml | 74 +++++++++++++++++------------- 1 file changed, 42 insertions(+), 32 deletions(-) diff --git a/bin/single-node/docker-compose.yml b/bin/single-node/docker-compose.yml index 5d4b7cc91a6..7363d72e49b 100644 --- a/bin/single-node/docker-compose.yml +++ b/bin/single-node/docker-compose.yml @@ -32,6 +32,31 @@ services: volumes: - minio_data:/data command: server --console-address ":9001" /data + healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:9000/minio/health/live"] + interval: 5s + timeout: 3s + retries: 10 + + # One-shot init container that creates the Iceberg warehouse bucket on first + # startup. MinIO's server image has no built-in "create bucket on boot" flag, + # so this sidecar uses the official `mc` client to do it idempotently. + minio-init: + image: minio/mc:RELEASE.2025-02-28T09-55-16Z + container_name: texera-minio-init + depends_on: + minio: + condition: service_healthy + env_file: + - .env + restart: "no" + entrypoint: ["/bin/sh", "-c"] + command: + - | + set -e + mc alias set local "$$STORAGE_S3_ENDPOINT" "$$STORAGE_S3_AUTH_USERNAME" "$$STORAGE_S3_AUTH_PASSWORD" + mc mb --ignore-existing "local/$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET" + echo "MinIO bucket '$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET' is ready." # PostgreSQL with PGroonga extension for full-text search. # Used by lakeFS and Texera's metadata storage. @@ -119,14 +144,16 @@ services: retries: 10 start_period: 10s + # One-shot init container that creates the Lakekeeper default project and + # the Iceberg warehouse pointing at the MinIO bucket prepared by minio-init. lakekeeper-init: image: alpine:3.19 container_name: texera-lakekeeper-init depends_on: lakekeeper: condition: service_healthy - minio: - condition: service_started + minio-init: + condition: service_completed_successfully env_file: - .env restart: "no" @@ -134,14 +161,10 @@ services: command: - | set -e - + echo "Installing dependencies..." apk add --no-cache curl ca-certificates - - echo "Installing MinIO Client..." - wget -q https://dl.min.io/client/mc/release/linux-amd64/mc -O /usr/local/bin/mc - chmod +x /usr/local/bin/mc - + check_status() { if [ "$$1" -ge 200 ] && [ "$$1" -lt 300 ]; then echo "Created $$2 successfully (HTTP $$1)." @@ -155,37 +178,24 @@ services: exit 1 fi } - - echo "Step 1: Initializing MinIO bucket '$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET'..." - mc alias set minio "$$STORAGE_S3_ENDPOINT" "$$STORAGE_S3_AUTH_USERNAME" "$$STORAGE_S3_AUTH_PASSWORD" || true - if mc ls minio/$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET > /dev/null 2>&1; then - echo "MinIO bucket '$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET' already exists." - else - mc mb minio/$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET || { - echo "Failed to create MinIO bucket '$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET'" - exit 1 - } - echo "MinIO bucket '$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET' created successfully." - fi - - - echo "Step 2: Initializing Default Project..." + + echo "Step 1: Initializing Default Project..." PROJECT_PAYLOAD='{"project-id": "00000000-0000-0000-0000-000000000000", "project-name": "default"}' - + PROJECT_CODE=$$(curl -s -o /tmp/response.txt -w "%{http_code}" \ -X POST \ -H "Content-Type: application/json" \ -d "$$PROJECT_PAYLOAD" \ "$$LAKEKEEPER_BASE_URI/management/v1/project" || echo "000") - + check_status "$$PROJECT_CODE" "Default Project" - - - echo "Step 3: Initializing Warehouse '$$STORAGE_ICEBERG_CATALOG_REST_WAREHOUSE_NAME'..." + + + echo "Step 2: Initializing Warehouse '$$STORAGE_ICEBERG_CATALOG_REST_WAREHOUSE_NAME'..." CREATE_PAYLOAD=$$(cat < Date: Wed, 15 Apr 2026 13:15:19 -0700 Subject: [PATCH 4/9] fix one issue --- bin/single-node/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/single-node/docker-compose.yml b/bin/single-node/docker-compose.yml index 7363d72e49b..9a1efaf8aa4 100644 --- a/bin/single-node/docker-compose.yml +++ b/bin/single-node/docker-compose.yml @@ -42,7 +42,7 @@ services: # startup. MinIO's server image has no built-in "create bucket on boot" flag, # so this sidecar uses the official `mc` client to do it idempotently. minio-init: - image: minio/mc:RELEASE.2025-02-28T09-55-16Z + image: minio/mc:RELEASE.2025-05-21T01-59-54Z container_name: texera-minio-init depends_on: minio: From 708d97df0f1c0696d25f338aec1587319899d499 Mon Sep 17 00:00:00 2001 From: mengw15 <125719918+mengw15@users.noreply.github.com> Date: Wed, 15 Apr 2026 13:46:15 -0700 Subject: [PATCH 5/9] fix one issue --- bin/single-node/docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/bin/single-node/docker-compose.yml b/bin/single-node/docker-compose.yml index 9a1efaf8aa4..ef1592f395c 100644 --- a/bin/single-node/docker-compose.yml +++ b/bin/single-node/docker-compose.yml @@ -135,8 +135,6 @@ services: - .env entrypoint: ["/home/nonroot/lakekeeper"] command: ["serve"] - ports: - - "8181:8181" healthcheck: test: ["CMD", "/home/nonroot/lakekeeper", "healthcheck"] interval: 10s From 3102b084b081ee35b7d0566eec23fd0dbaea84c2 Mon Sep 17 00:00:00 2001 From: mengw15 <125719918+mengw15@users.noreply.github.com> Date: Wed, 15 Apr 2026 14:13:41 -0700 Subject: [PATCH 6/9] fix one issue --- bin/single-node/docker-compose.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/bin/single-node/docker-compose.yml b/bin/single-node/docker-compose.yml index ef1592f395c..892276e6f28 100644 --- a/bin/single-node/docker-compose.yml +++ b/bin/single-node/docker-compose.yml @@ -219,7 +219,14 @@ services: -d "$$CREATE_PAYLOAD" \ "$$LAKEKEEPER_BASE_URI/management/v1/warehouse" || echo "000") - check_status "$$WAREHOUSE_CODE" "Lakekeeper Warehouse" + # Lakekeeper returns 400 CreateWarehouseStorageProfileOverlap when a + # warehouse with the same name + storage profile already exists. + # Treat that as success so this init is idempotent across restarts. + if [ "$$WAREHOUSE_CODE" = "400" ] && grep -q "CreateWarehouseStorageProfileOverlap" /tmp/response.txt 2>/dev/null; then + echo "Lakekeeper Warehouse already exists (storage profile overlap). Treating as success." + else + check_status "$$WAREHOUSE_CODE" "Lakekeeper Warehouse" + fi echo "Initialization sequence completed successfully!" From 665ddbf4eaf0b5398842362fb33a3849a6c75872 Mon Sep 17 00:00:00 2001 From: mengw15 <125719918+mengw15@users.noreply.github.com> Date: Mon, 20 Apr 2026 17:52:44 -0700 Subject: [PATCH 7/9] resolve comment, dependency --- bin/single-node/docker-compose.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/single-node/docker-compose.yml b/bin/single-node/docker-compose.yml index bfbccf44f4f..614a9ec33b6 100644 --- a/bin/single-node/docker-compose.yml +++ b/bin/single-node/docker-compose.yml @@ -160,8 +160,9 @@ services: - | set -e - echo "Installing dependencies..." - apk add --no-cache curl ca-certificates + # curl is used to POST to Lakekeeper's management API over the internal Docker network. + echo "Installing curl (required to call Lakekeeper's management API)..." + apk add --no-cache curl check_status() { if [ "$$1" -ge 200 ] && [ "$$1" -lt 300 ]; then From f1d14088ba7d753a62fc8b69ea980ae01ae2588e Mon Sep 17 00:00:00 2001 From: mengw15 <125719918+mengw15@users.noreply.github.com> Date: Mon, 20 Apr 2026 18:12:13 -0700 Subject: [PATCH 8/9] resolve comment, default project --- bin/single-node/docker-compose.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/single-node/docker-compose.yml b/bin/single-node/docker-compose.yml index 614a9ec33b6..036f8b5e1da 100644 --- a/bin/single-node/docker-compose.yml +++ b/bin/single-node/docker-compose.yml @@ -178,7 +178,12 @@ services: fi } - echo "Step 1: Initializing Default Project..." + # Lakekeeper organizes warehouses under "projects" (its top-level tenant + # abstraction). Texera is single-tenant, so we create one project using + # the NIL UUID (all zeros) — with LAKEKEEPER__ENABLE_DEFAULT_PROJECT=true + # (Lakekeeper's default), this is the project that any client request + # without a project-id is routed to. + echo "Step 1: Creating Lakekeeper's default project (top-level tenant that will hold the Iceberg warehouse)..." PROJECT_PAYLOAD='{"project-id": "00000000-0000-0000-0000-000000000000", "project-name": "default"}' PROJECT_CODE=$$(curl -s -o /tmp/response.txt -w "%{http_code}" \ From a64931c325e481ad1ce385d5c56786b86f8def88 Mon Sep 17 00:00:00 2001 From: mengw15 <125719918+mengw15@users.noreply.github.com> Date: Tue, 21 Apr 2026 01:05:47 -0700 Subject: [PATCH 9/9] resolve comment, update to pre check warehouse exists --- bin/single-node/docker-compose.yml | 49 +++++++++++++++++++----------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/bin/single-node/docker-compose.yml b/bin/single-node/docker-compose.yml index 036f8b5e1da..b6917f413d1 100644 --- a/bin/single-node/docker-compose.yml +++ b/bin/single-node/docker-compose.yml @@ -160,9 +160,11 @@ services: - | set -e - # curl is used to POST to Lakekeeper's management API over the internal Docker network. - echo "Installing curl (required to call Lakekeeper's management API)..." - apk add --no-cache curl + # curl is used to call Lakekeeper's management API over the internal + # Docker network; jq parses list responses so we can detect existing + # resources before deciding to create them (keeps init idempotent). + echo "Installing curl (to call Lakekeeper's management API) and jq (to parse its list responses)..." + apk add --no-cache curl jq check_status() { if [ "$$1" -ge 200 ] && [ "$$1" -lt 300 ]; then @@ -195,8 +197,27 @@ services: check_status "$$PROJECT_CODE" "Default Project" - echo "Step 2: Initializing Warehouse '$$STORAGE_ICEBERG_CATALOG_REST_WAREHOUSE_NAME'..." - CREATE_PAYLOAD=$$(cat </dev/null; then + echo "Lakekeeper Warehouse '$$STORAGE_ICEBERG_CATALOG_REST_WAREHOUSE_NAME' already exists. Skipping creation." + else + echo "Warehouse not found. Creating '$$STORAGE_ICEBERG_CATALOG_REST_WAREHOUSE_NAME'..." + CREATE_PAYLOAD=$$(cat </dev/null; then - echo "Lakekeeper Warehouse already exists (storage profile overlap). Treating as success." - else check_status "$$WAREHOUSE_CODE" "Lakekeeper Warehouse" fi