diff --git a/charts/postgres-operator-ui/values.yaml b/charts/postgres-operator-ui/values.yaml
index da3c4baafd948b03731128fdb45748420a7bc63c..9923ff0231517cc797ce8b09b3a481f2835341cc 100644
--- a/charts/postgres-operator-ui/values.yaml
+++ b/charts/postgres-operator-ui/values.yaml
@@ -62,8 +62,6 @@ podAnnotations:
extraEnvs:
[]
# Exemple of settings to make snapshot view working in the ui when using AWS
- # - name: WALE_S3_ENDPOINT
- # value: https+path://s3.us-east-1.amazonaws.com:443
# - name: SPILO_S3_BACKUP_PREFIX
# value: spilo/
# - name: AWS_ACCESS_KEY_ID
@@ -83,8 +81,6 @@ extraEnvs:
# key: AWS_DEFAULT_REGION
# - name: SPILO_S3_BACKUP_BUCKET
# value: <s3 bucket used by the operator>
- # - name: "USE_AWS_INSTANCE_PROFILE"
- # value: "true"
# configure UI service
service:
diff --git a/ui/manifests/deployment.yaml b/ui/manifests/deployment.yaml
index e09dd1e4f0ea36414f6400fb7e39ca318588ff7a..3b30974167fd9137138df31bb810bf3e6b4e82c1 100644
--- a/ui/manifests/deployment.yaml
+++ b/ui/manifests/deployment.yaml
@@ -81,8 +81,6 @@ spec:
]
}
# Exemple of settings to make snapshot view working in the ui when using AWS
- # - name: WALE_S3_ENDPOINT
- # value: https+path://s3.us-east-1.amazonaws.com:443
# - name: SPILO_S3_BACKUP_PREFIX
# value: spilo/
# - name: AWS_ACCESS_KEY_ID
@@ -102,5 +100,3 @@ spec:
# key: AWS_DEFAULT_REGION
# - name: SPILO_S3_BACKUP_BUCKET
# value: <s3 bucket used by the operator>
- # - name: "USE_AWS_INSTANCE_PROFILE"
- # value: "true"
diff --git a/ui/operator_ui/main.py b/ui/operator_ui/main.py
index e02c2995cbccc16a21e53dcf771b2da22af6f824..bf28df6eb128c160cc79a04bb4547b62a60ac549 100644
--- a/ui/operator_ui/main.py
+++ b/ui/operator_ui/main.py
@@ -95,14 +95,6 @@ DEFAULT_MEMORY_LIMIT = getenv('DEFAULT_MEMORY_LIMIT', '300Mi')
DEFAULT_CPU = getenv('DEFAULT_CPU', '10m')
DEFAULT_CPU_LIMIT = getenv('DEFAULT_CPU_LIMIT', '300m')
-WALE_S3_ENDPOINT = getenv(
- 'WALE_S3_ENDPOINT',
- 'https+path://s3.eu-central-1.amazonaws.com:443',
-)
-
-USE_AWS_INSTANCE_PROFILE = (
- getenv('USE_AWS_INSTANCE_PROFILE', 'false').lower() != 'false'
-)
AWS_ENDPOINT = getenv('AWS_ENDPOINT')
@@ -784,8 +776,6 @@ def get_versions(pg_cluster: str):
bucket=SPILO_S3_BACKUP_BUCKET,
pg_cluster=pg_cluster,
prefix=SPILO_S3_BACKUP_PREFIX,
- s3_endpoint=WALE_S3_ENDPOINT,
- use_aws_instance_profile=USE_AWS_INSTANCE_PROFILE,
),
)
@@ -797,9 +787,8 @@ def get_basebackups(pg_cluster: str, uid: str):
bucket=SPILO_S3_BACKUP_BUCKET,
pg_cluster=pg_cluster,
prefix=SPILO_S3_BACKUP_PREFIX,
- s3_endpoint=WALE_S3_ENDPOINT,
uid=uid,
- use_aws_instance_profile=USE_AWS_INSTANCE_PROFILE,
+ postgresql_versions=OPERATOR_UI_CONFIG.get('postgresql_versions', DEFAULT_UI_CONFIG['postgresql_versions']),
),
)
@@ -991,8 +980,6 @@ def main(port, debug, clusters: list):
logger.info(f'Superuser team: {SUPERUSER_TEAM}')
logger.info(f'Target namespace: {TARGET_NAMESPACE}')
logger.info(f'Teamservice URL: {TEAM_SERVICE_URL}')
- logger.info(f'Use AWS instance_profile: {USE_AWS_INSTANCE_PROFILE}')
- logger.info(f'WAL-E S3 endpoint: {WALE_S3_ENDPOINT}')
logger.info(f'AWS S3 endpoint: {AWS_ENDPOINT}')
if TARGET_NAMESPACE is None:
diff --git a/ui/operator_ui/spiloutils.py b/ui/operator_ui/spiloutils.py
index f715430a1da46620e752431a79c8e79f37f141d7..6a2f03bb2bbb94b1a15cb4d13b35b79b15104e85 100644
--- a/ui/operator_ui/spiloutils.py
+++ b/ui/operator_ui/spiloutils.py
@@ -6,9 +6,8 @@ from os import environ, getenv
from requests import Session
from urllib.parse import urljoin
from uuid import UUID
-from wal_e.cmd import configure_backup_cxt
-from .utils import Attrs, defaulting, these
+from .utils import defaulting, these
from operator_ui.adapters.logger import logger
session = Session()
@@ -284,10 +283,8 @@ def read_stored_clusters(bucket, prefix, delimiter='/'):
def read_versions(
pg_cluster,
bucket,
- s3_endpoint,
prefix,
delimiter='/',
- use_aws_instance_profile=False,
):
return [
'base' if uid == 'wal' else uid
@@ -305,35 +302,72 @@ def read_versions(
if uid == 'wal' or defaulting(lambda: UUID(uid))
]
-BACKUP_VERSION_PREFIXES = ['', '10/', '11/', '12/', '13/', '14/', '15/', '16/', '17/']
+def lsn_to_wal_segment_stop(finish_lsn, start_segment, wal_segment_size=16 * 1024 * 1024):
+ timeline = int(start_segment[:8], 16)
+ log_id = finish_lsn >> 32
+ seg_id = (finish_lsn & 0xFFFFFFFF) // wal_segment_size
+ return f"{timeline:08X}{log_id:08X}{seg_id:08X}"
+
+def lsn_to_offset_hex(lsn, wal_segment_size=16 * 1024 * 1024):
+ return f"{lsn % wal_segment_size:08X}"
def read_basebackups(
pg_cluster,
uid,
bucket,
- s3_endpoint,
prefix,
- delimiter='/',
- use_aws_instance_profile=False,
+ postgresql_versions,
):
- environ['WALE_S3_ENDPOINT'] = s3_endpoint
suffix = '' if uid == 'base' else '/' + uid
backups = []
- for vp in BACKUP_VERSION_PREFIXES:
-
- backups = backups + [
- {
- key: value
- for key, value in basebackup.__dict__.items()
- if isinstance(value, str) or isinstance(value, int)
- }
- for basebackup in Attrs.call(
- f=configure_backup_cxt,
- aws_instance_profile=use_aws_instance_profile,
- s3_prefix=f's3://{bucket}/{prefix}{pg_cluster}{suffix}/wal/{vp}',
- )._backup_list(detail=True)
- ]
+ for vp in postgresql_versions:
+ backup_prefix = f'{prefix}{pg_cluster}{suffix}/wal/{vp}/basebackups_005/'
+ logger.info(f"{bucket}/{backup_prefix}")
+
+ paginator = client('s3').get_paginator('list_objects_v2')
+ pages = paginator.paginate(Bucket=bucket, Prefix=backup_prefix)
+
+ for page in pages:
+ for obj in page.get("Contents", []):
+ key = obj["Key"]
+ if not key.endswith("backup_stop_sentinel.json"):
+ continue
+
+ response = client('s3').get_object(Bucket=bucket, Key=key)
+ backup_info = loads(response["Body"].read().decode("utf-8"))
+ last_modified = response["LastModified"].astimezone(timezone.utc).isoformat()
+
+ backup_name = key.split("/")[-1].replace("_backup_stop_sentinel.json", "")
+ start_seg, start_offset = backup_name.split("_")[1], backup_name.split("_")[-1] if "_" in backup_name else None
+
+ if "LSN" in backup_info and "FinishLSN" in backup_info:
+ # WAL-G
+ lsn = backup_info["LSN"]
+ finish_lsn = backup_info["FinishLSN"]
+ backups.append({
+ "expanded_size_bytes": backup_info.get("UncompressedSize"),
+ "last_modified": last_modified,
+ "name": backup_name,
+ "wal_segment_backup_start": start_seg,
+ "wal_segment_backup_stop": lsn_to_wal_segment_stop(finish_lsn, start_seg),
+ "wal_segment_offset_backup_start": lsn_to_offset_hex(lsn),
+ "wal_segment_offset_backup_stop": lsn_to_offset_hex(finish_lsn),
+ })
+ elif "wal_segment_backup_stop" in backup_info:
+ # WAL-E
+ stop_seg = backup_info["wal_segment_backup_stop"]
+ stop_offset = backup_info["wal_segment_offset_backup_stop"]
+
+ backups.append({
+ "expanded_size_bytes": backup_info.get("expanded_size_bytes"),
+ "last_modified": last_modified,
+ "name": backup_name,
+ "wal_segment_backup_start": start_seg,
+ "wal_segment_backup_stop": stop_seg,
+ "wal_segment_offset_backup_start": start_offset,
+ "wal_segment_offset_backup_stop": stop_offset,
+ })
return backups
diff --git a/ui/requirements.txt b/ui/requirements.txt
index d3318ceecbfe1edde1f5cd92d156612b82013ebd..783c0aac377cceb13bf8d59f6b4f0a187dbaf833 100644
--- a/ui/requirements.txt
+++ b/ui/requirements.txt
@@ -11,5 +11,4 @@ kubernetes==11.0.0
python-json-logger==2.0.7
requests==2.32.2
stups-tokens>=1.1.19
-wal_e==1.1.1
werkzeug==3.0.6